In [8]: next(gen)                                                                                                                                                    
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/mongo_core.py in get_events_generator(descriptor, event_col, descriptor_col, descriptor_cache, run_start_col, run_start_cache, convert_arrays)
     64             try:
---> 65                 _dk = data_keys[k]
     66             except KeyError as err:

KeyError: 'fs'

The above exception was the direct cause of the following exception:

MismatchedDataKeys                        Traceback (most recent call last)
<ipython-input-8-6e72e47198db> in <module>
----> 1 next(gen)

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/_core.py in documents(self, stream_name, fields, fill)
    324                                     stream_name=stream_name,
    325                                     fill=fill)
--> 326         for payload in gen:
    327             yield payload
    328 

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/_core.py in get_documents(self, headers, stream_name, fields, fill, handler_registry)
   1404                         header=h, stream_name=stream_name,
   1405                         fields=fields)
-> 1406                     for name, doc in gen:
   1407                         # Find all the res/datum backed keys
   1408                         if name == 'descriptor':

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in docs_given_header(self, header, stream_name, fields)
    157             yield 'descriptor', d
    158             ev_gens.append(self.mds.get_events_generator(d))
--> 159         for ev in interlace_gens(*ev_gens):
    160             event_data = ev['data']  # cache for perf
    161             desc = ev['descriptor']

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in interlace_gens(*gens)
     45 
     46     for j, itr in enumerate(iters):
---> 47         safe_next(itr, j)
     48     while heap:
     49         _, j, val, itr = heapq.heappop(heap)

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in safe_next(itr, indx)
     39     def safe_next(itr, indx):
     40         try:
---> 41             val = next(itr)
     42         except StopIteration:
     43             return

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/base.py in get_events_generator(self, descriptor, convert_arrays)
    191         # when we drop 2.7, this can be
    192         # yield from evs
--> 193         for ev in evs:
    194             yield ev
    195 

/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/mongo_core.py in get_events_generator(descriptor, event_col, descriptor_col, descriptor_cache, run_start_col, run_start_cache, convert_arrays)
     73                     f"event['data'].keys(): {ev['data'].keys()}, "
     74                     "descriptor['data_keys'].keys(): "
---> 75                     f"{descriptor['data_keys'].keys()}") from err
     76             # convert any arrays stored directly in mds into ndarray
     77             if convert_arrays:

MismatchedDataKeys: The documents are not valid.  Either because they were recorded incorrectly in the first place, corrupted since, or exercising a yet-undiscovered bug in a reader. event['data'].keys() must equal descriptor['data_keys'].keys(). event['data'].keys(): dict_keys(['pe1c_image', 'pe1c_stats1_total', 'start_Grid_X', 'stop_Grid_X', 'Grid_Y', 'Grid_Y_user_setpoint', 'fs']), descriptor['data_keys'].keys(): dict_keys(['start_Grid_X', 'stop_Grid_X', 'pe1c_image', 'pe1c_stats1_total', 'Grid_Y', 'Grid_Y_user_setpoint'])